#coding=utf8
import os
import sys
import numpy as np
import pandas as pd
import mxnet as mx
import cPickle
from collections import namedtuple
Batch = namedtuple('Batch', ['data'])
sys.path.append('../utils/')
sys.path.append('../')
from mxnet_helper import get_image
from concat_ft import mx_extract_ft
from sklearn.preprocessing import StandardScaler,LabelEncoder


k_folds= 7
TRAIN_FOLDER = '/media/hszc/data/BaiduImage/dataset/trainall/'
model_folder = '/media/hszc/model/zhangchi/BaiduImage_model/'
vallst_path = '../data/trainall_val.lst'
model_name = 'Resnet50'
layer_name = 'flatten0'
devs = mx.gpu(0)

df = pd.read_csv(vallst_path,header=None,sep='\t',names=['idx','mxlabel','img_path'])
img_path_list_val = list(TRAIN_FOLDER + df['img_path'].str[:])
y_val = df['img_path'].str.split('/').str[-2]


X_train = []
truelabel = []
mxlabel = []
temp_dict = {}
#preprare train dataset
for i in range(k_folds):
    mx_ckpt_epoch = 24
    model_ckpt_path = '{0}/{1}/fold_{2}_finetune/{1}'.format(model_folder,model_name,i)
    df = pd.read_csv('{0}_fold_data/trainall_test{1}.lst'.format(k_folds,i),header=None,sep='\t',names=['idx','mxlabel','img_path'])
    img_path_list =  list(TRAIN_FOLDER + df['img_path'].str[:])
    ft_list_train = mx_extract_ft(model_ckpt_path, mx_ckpt_epoch, layer_name, img_path_list)

    X_train += ft_list_train
    truelabel += list(df['img_path'].str.split('/').str[-2])

    ft_list_val = mx_extract_ft(model_ckpt_path, mx_ckpt_epoch, layer_name, img_path_list_val)
    temp_dict[i] = np.array(ft_list_val)


X_train = np.array(X_train)
le = LabelEncoder()
y_train = le.fit_transform(truelabel)
y_val = le.transform(y_val)

scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)


# train a meta clf
import xgboost as xgb
params={
	'booster':'gbtree',
	'objective':'multi:softprob',
	'gamma':0.1,
	'max_depth':5,
	# 'lambda':70,
	'subsample':0.7,
	'colsample_bytree':0.3,
	'min_child_weight':0.3,
	'eta': 0.04,
	'seed':69,
	'silent':1,
    'num_class':100,
	}
#
dtrain = xgb.DMatrix(X_train, y_train)
# clf_xgb = xgb.cv(params, dtrain, num_boost_round=500 , nfold=5,verbose_eval=True)
num_boost_round = 50  #71
clf = xgb.train(params, dtrain, num_boost_round)#,evals=watchlist)
cPickle.dump(clf, open('./blending_ft_model/xgb.pkl', 'wb'))
y_pred_train = np.round(clf.predict(xgb.DMatrix(X_train)))
y_pred_train = np.argmax(y_pred_train,axis=1)
# val
y_pred = np.empty((k_folds,y_val.shape[0],100))
for i in range(k_folds):
	X_val = scaler.transform(temp_dict[i])
	y_pred[i] = np.round(clf.predict(xgb.DMatrix(X_val)))


y_pred = np.sum(y_pred,axis=0)
y_pred = np.argmax(y_pred,axis=1)
print y_pred.shape

from sklearn.metrics import classification_report,accuracy_score
print accuracy_score(y_train, y_pred_train, normalize=True, sample_weight=None),X_train.shape
print accuracy_score(y_val, y_pred, normalize=True, sample_weight=None),X_val.shape


#online test



